package in.ac.iitb.cse.nlp.postagger.data;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.util.ArrayList;
import java.util.List;

public class FiveFold {
	TransitionMatrix tm;
	EmissionMatrix em;
	List<String> ts;
	TableCreator tc;

	List<String> FileList;
	List<String> training;
	List<String> testing;

	HMM vhmm = null;
	
	long tstart, tcurrent, tend;
	File fInfo;
	BufferedWriter wInfo;
	int nRun;
	int nFold;
	String fPrefix="";
	private long totalNoOfSentences;
	private long totalNoOfCorrectSentences;
	
	ConfusionMatrix confmat;

	public FiveFold(int fold, String prefix) throws IOException {
		FileList = new ArrayList<String>();
		training = new ArrayList<String>();
		testing = new ArrayList<String>();
		FileList = new ArrayList<String>();
		
		nFold = fold;
		fPrefix = prefix;

		String fname = fPrefix+"info.txt";
	    fInfo = new File(fname);
		wInfo = new BufferedWriter(new OutputStreamWriter(
				new FileOutputStream(fInfo)));
		
		noOfPredictions = noOfCorrectPredictions = 0;
		totalNoOfCorrectPredictions = totalNoOfPredictions = 0;

		noOfSentences = noOfCorrectSentences = 0;
		totalNoOfCorrectSentences = totalNoOfSentences = 0;

		tstart = System.currentTimeMillis()/1000;
		
		fiveFold("", "corpus");
		
		
		return;
	}
	
	public void init()
	{
		if (Config.isTrigram)
		{
			tc = new TriTableCreator();
		}
		else
		{
			tc = new BiTableCreator();
		}		
		DataMaps.tags.clear();
		DataMaps.words.clear();
	}
	
	public TableCreator newTraining() {
		TableCreator tc = null;
		if (Config.isTrigram)
		{
			tc = new TriTableCreator();
		}
		else
			tc = new BiTableCreator();
		// tableCreator.go();
		/*
		 * tableCreator.processLine("PNP_I VVB_will PUN_. PNP_I VVB_will PUN_.");
		 * System.out.println(TableCreator.transitionMatrix.toString());
		 * System.out.println(TableCreator.emissionMatrix.toString());
		 * List<String> b =tableCreator.tagSentence("^ I will $");
		 * System.out.println(b.toString());
		 */
		DataMaps.tags.clear();
		DataMaps.words.clear();

		/*
		 * tc.goDir("", "corpus");
		 * 
		 * tm =tc.transitionMatrix; em =tc.emissionMatrix; ts = new
		 * ArrayList<String>(tc.tagSet) ;
		 * 
		 * Config.nTotalTags = tc.tagSet.size();
		 */
		return tc;
	}
	
	public File[] getTesting(int fold,int run, File [] f)
	{
		int n = f.length;
		int l=n/fold;
		int start=run*l;
		int end = run*l + l-1;
		if( end >= n) end = n-1;
		File r[] = new File[end-start+1];
		for(int i=start,j=0; i<=end;i++,j++)
			r[j] = f[i];
		return r;
	}

	public File[] getTraining(int fold,int run, File [] f)
	{
		int n = f.length;
		int l=n/fold;
		int start=run*l;
		int end = run*l + l-1;
		if( end >= n) end = n-1;
		File r[] = new File[n-end+start-1];
		int j=0;
		for(int i=0; i<start;i++,j++)
			r[j] = f[i];
		for(int i=end+1; i<n;i++,j++)
			r[j] = f[i];
		return r;
	}
	
	void printInfo(String str) throws IOException
	{
		long tdiff = System.currentTimeMillis()/1000 - tstart;
		wInfo.write(""+tdiff+"|R"+nRun+"|"+str+"|\n");
	}
	
	public double fiveFold(String cdir, String dir) throws IOException {
				
		if (cdir.endsWith("/") == false && dir.endsWith("/") == false)
			cdir.concat("/");
		File Dir = new File(cdir + dir);
		if (Dir.isDirectory()) {
			//First do training with the 80% of the files
			confmat = new ConfusionMatrix();
			
			for(int Run=0;Run<nFold;Run++)
			{
				
				init();
					
				noOfSentences = noOfCorrectSentences = 0;
				noOfPredictions=noOfCorrectPredictions=0;
				nRun = Run;
				
				Config.nTrainWords = 0;
				Config.nTrainTags = 0;
				
				printInfo("INFO|Run "+Run+" Started..");
				System.out.println("INFO|Run "+Run+" Started..");
				
				ConfusionMatrix cm = new ConfusionMatrix();

				File[] Files = Dir.listFiles(new FilenameFilter() {
					public boolean accept(File dir, String name) {
						if(name.endsWith("txt"))
							return true;
						return false;
					}
				});
			
				File tf[] = getTraining(nFold,Run, Files);
				for (int i = 0; i < tf.length; i++) {
					File file = tf[i];
					if (file.isFile() && file.canRead()) {
						tc.goFile(file.getAbsolutePath());
						printInfo("TRAIN| "+file.getName());
					}
				}

				tm = tc.transitionMatrix;
				em = tc.emissionMatrix;
				ts = new ArrayList<String>(tc.tagSet);
				Config.nTotalTags = tc.tagSet.size();
			
				//Calc Smoothing Factors for trigrams
				if(Config.isTrigram)
				{
					TriViterbiHMM.calcSmoothingFactors(tc.transitionMatrix, new ArrayList<String>(tc.tagSet));
					printInfo("SMOOTH|"+Config.h[0]+"|"+Config.h[1]+"|"+Config.h[2]);
				}
				
				//Config.h[0]=Config.h[1]=(double) 0;
				//Config.h[2]=(double) 1;
				
				if (Config.isTrigramInUse)
					vhmm = new TriViterbiHMM();
				else
					vhmm = new BiViterbiHMM();

				// Now test with 20% of the files whether the tags
				// are correct
				File testf[] = getTesting(nFold,Run, Files);
				for (int i = 0; i < testf.length; i++) {
					if(testf[i].getName().endsWith("txt") == false ) continue;
					File file = testf[i];
					currTestFile = testf[i].getAbsolutePath();
/* CREATING TAGGED COMPLETE FILE IS DISABLED					
					File fout = new File(currTestFile + ".out"); // Clear the output
					File f = new File(currTestFile + ".out");
					if (fout != null)
						fout.delete();
					writer = new BufferedWriter(new FileWriter(f, true));
*/					
					if (file.isFile() && file.canRead()) {
						test(file,cm);
						printInfo("TEST| "+file.getName());
					}
//					System.out.println(file.getAbsolutePath() + " used for testing");
/* CREATING TAGGED COMPLETE FILE IS DISABLED					
					writer.close();
*/	
				}
				
				String fmFile=fPrefix+"Run_"+Run+"_accuracy.csv";
				cm.producePosAccuracyFinal(fmFile);
				String cmFile=fPrefix+"Run_"+Run+"_confmatix.csv";
				cm.produceConfMatrix(cmFile);
				printInfo("INFO|Run "+Run+" Finished..");
				printInfo("ACCU|"+noOfCorrectPredictions+"|"+noOfPredictions+" = "+(noOfCorrectPredictions*100.0/noOfPredictions));
				printInfo("ACCUSEN|"+noOfCorrectSentences+"|"+noOfSentences+" = "+(noOfCorrectSentences*100.0/noOfSentences));
			
				totalNoOfCorrectPredictions += noOfCorrectPredictions;
				totalNoOfPredictions += noOfPredictions;
				totalNoOfCorrectSentences += noOfCorrectSentences;
				totalNoOfSentences += noOfSentences;
				
			} // end for Runs
			String fmFile=fPrefix+"accuracy.csv";
			confmat.producePosAccuracyFinal(fmFile);
			String cmFile=fPrefix+"confmatix.csv";
			confmat.produceConfMatrix(cmFile);			
		}//end If directory
		
		printInfo("TRAINWORDS|"+Config.nTrainWords);
		printInfo("TACCU|"+totalNoOfCorrectPredictions+"|"+totalNoOfPredictions+"|"+(totalNoOfCorrectPredictions*100.0/totalNoOfPredictions));
		printInfo("TACCUSEN|"+totalNoOfCorrectSentences+"|"+totalNoOfSentences+"|"+(totalNoOfCorrectSentences*100.0/totalNoOfSentences));
		
		tend = System.currentTimeMillis()/1000;
		printInfo("TIME|"+(tend-tstart)/60+" minutes,"+(tend-tstart)%60+" seconds");
		
		wInfo.close();
		return (((double) noOfCorrectPredictions) / (double) (noOfPredictions));
	}

	private void test(File file, ConfusionMatrix cm) throws IOException {
		BufferedReader reader = new BufferedReader(new InputStreamReader(
				new FileInputStream(file)));
		String line = null;
		while ((line = reader.readLine()) != null) {
			String[] split = line.split("PUN" + TAG_SEPARATOR + END_OF_SENTENCE);
			for (String string : split) {
				String[] split2 = string.split("PUN" + TAG_SEPARATOR + ALTERNATE_END_OF_SENTENCE);
				if (split2.length > 1) {
					predict(split2[0], "PUN" + TAG_SEPARATOR + ALTERNATE_END_OF_SENTENCE, cm);
					predict(split2[1], "PUN" + TAG_SEPARATOR + END_OF_SENTENCE,cm);
				} else {
					predict(split2[0], "PUN" + TAG_SEPARATOR + END_OF_SENTENCE,cm);
				}
			}
		}
	}

	private void predict(String string, String endOfSentence, ConfusionMatrix cm)
			throws IOException {
		string = string.trim() + " " + endOfSentence;
		String[] split = string.trim().split(WORD_SEPARATOR);
		List<String> actualTagSequence = new ArrayList<String>();
		List<String> sentence = new ArrayList<String>();
		for (String splitString : split) {
			String[] split2 = splitString.split(TAG_SEPARATOR);
			actualTagSequence.add(split2[0]);
			sentence.add(split2[1]);
		}

		vhmm.init(ts, tm, em);

		List<String> bestProbTagSeq = vhmm.bestProbTagSeq(sentence);

		int wordOccurActual=0;
		int wordOccurPredict=0;
		
		if (bestProbTagSeq != null
				&& bestProbTagSeq.size() == actualTagSequence.size()) {
			noOfPredictions+=actualTagSequence.size();
			noOfSentences++;
			long expCorrectPred = noOfCorrectPredictions + actualTagSequence.size();
			long unk_count=0;
			for (int i = 0; i < actualTagSequence.size(); i++) {
				String act = actualTagSequence.get(i);
				String best = bestProbTagSeq.get(i);
				String word = sentence.get(i);
				wordOccurPredict = em.getCount(best, word);
				wordOccurActual = em.getCount(act, word);
				String info = word+"|"+act+"|"+best+"|P"+wordOccurPredict+"|A"+wordOccurActual+"|ST"+noOfSentences;
				if(wordOccurPredict ==0)
				{
					unk_count++;
					info += "|UNK"+unk_count;
				}
				if( act.contains("-"))
				{
					if(act.contains(best))
					{
						cm.update(best, best);
						confmat.update(best, best);
						noOfCorrectPredictions++;
						printInfo("ACTAG|"+info);
					}
					else
					{
						cm.update(act, best);
						confmat.update(act, best);
						printInfo("AWTAG|"+info);
					}						
				}
				else
				{
					cm.update(act, best);
					confmat.update(act, best);
					if (act.equals(best)) {
						noOfCorrectPredictions++;
						printInfo("XCTAG|"+ info);
					}else
					{
						printInfo("XWTAG|"+info);
					}
				}				
				/* CREATING TAGGED COMPLETE FILE IS DISABLED					
					writer.write(" " + bestProbTagSeq.get(i) + "_" + sentence.get(i));
				*/				
			}//end for
			if(expCorrectPred == noOfCorrectPredictions)
			noOfCorrectSentences++; 
		}// end if act.size == best.size
		else{
			//System.out.println(sentence.toString());
			//System.out.println(actualTagSequence.toString());
			//System.out.println(bestProbTagSeq.toString());
			//System.out.println("oHH!!! How can you be here !!!!!!");
			//System.out.println(""+actualTagSequence.size()+" "+bestProbTagSeq.size());
		}
	}

	// For accuracy testing
	long noOfPredictions;
	long noOfCorrectPredictions;
	long totalNoOfPredictions;
	long totalNoOfCorrectPredictions;
	long noOfCorrectSentences;
	long noOfSentences;
	

	static final String WORD_SEPARATOR = " ";
	static final String TAG_SEPARATOR = "_";
	// We will divide the input into sentences so that
	// we can process it faster
	static final String START_OF_SENTENCE = "^";
	static final String END_OF_SENTENCE = ".";
	static final String ALTERNATE_END_OF_SENTENCE = "?";
	String currTestFile;
	BufferedWriter writer;

	public static void main(String args[]) throws IOException {
		FiveFold f;
		int folds = 5;

		Config.isTrigram = false;
		Config.isTrigramInUse = false;
		Config.isBigram = true;
		Config.isBigramInUse = true;
		Config.isAmbigutyResolv = true;

		
		Config.isEmiIgnore = false;
		Config.isTransLapSmoothing = false;
		Config.isEmiLapSmoothing = false;
		f = new FiveFold(folds,"op_bi_aa_");
		System.out.println("Total Acc="+f.totalNoOfCorrectPredictions +"/"+ f.totalNoOfPredictions);
		System.out.println("Total Acc="+f.totalNoOfCorrectPredictions*100.0/f.totalNoOfPredictions);
		

		Config.isEmiIgnore = true;
		Config.isTransLapSmoothing = false;
		Config.isEmiLapSmoothing = false;
		f = new FiveFold(folds,"op_bi_emi_");
		System.out.println("Total Acc="+f.totalNoOfCorrectPredictions +"/"+ f.totalNoOfPredictions);
		System.out.println("Total Acc="+f.totalNoOfCorrectPredictions*100.0/f.totalNoOfPredictions);
		
		Config.isEmiIgnore = false;
		Config.isTransLapSmoothing = false;
		Config.isEmiLapSmoothing = true;
		f = new FiveFold(folds,"op_bi_lap_");
		System.out.println("Total Acc="+f.totalNoOfCorrectPredictions +"/"+ f.totalNoOfPredictions);
		System.out.println("Total Acc="+f.totalNoOfCorrectPredictions*100.0/f.totalNoOfPredictions);

	
	/*
		Config.isTrigram = true;
		Config.isTrigramInUse = true;
		Config.isBigram = false;
		Config.isBigramInUse = false;
		Config.isAmbigutyResolv = true;

		Config.isEmiIgnore = false;
		Config.isTransLapSmoothing = false;
		Config.isEmiLapSmoothing = false;
		f = new FiveFold(folds,"op_tri_aa_");
		System.out.println("Total Acc="+f.totalNoOfCorrectPredictions +"/"+ f.totalNoOfPredictions);
		System.out.println("Total Acc="+f.totalNoOfCorrectPredictions*100.0/f.totalNoOfPredictions);

		
		Config.isEmiIgnore = true;
		Config.isTransLapSmoothing = false;
		Config.isEmiLapSmoothing = false;
		f = new FiveFold(folds,"op_tri_emi_");
		System.out.println("Total Acc="+f.totalNoOfCorrectPredictions +"/"+ f.totalNoOfPredictions);
		System.out.println("Total Acc="+f.totalNoOfCorrectPredictions*100.0/f.totalNoOfPredictions);
		
		Config.isEmiIgnore = false;
		Config.isTransLapSmoothing = false;
		Config.isEmiLapSmoothing = true;
		f = new FiveFold(folds,"op_tri_lap_");
		System.out.println("Total Acc="+f.totalNoOfCorrectPredictions +"/"+ f.totalNoOfPredictions);
		System.out.println("Total Acc="+f.totalNoOfCorrectPredictions*100.0/f.totalNoOfPredictions);
		
	*/
	}
}
